blob: de12fd745050c122eaf0e145bffd334dbaac2782 [file] [log] [blame]
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.drill.exec.compile;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import org.apache.drill.common.util.DrillFileUtils;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.compile.bytecode.ValueHolderReplacementVisitor;
import org.apache.drill.test.BaseTestQuery;
import org.apache.drill.exec.compile.ClassTransformer.ClassSet;
import org.apache.drill.exec.compile.sig.GeneratorMapping;
import org.apache.drill.exec.compile.sig.MappingSet;
import org.apache.drill.exec.exception.ClassTransformationException;
import org.apache.drill.exec.expr.ClassGenerator;
import org.apache.drill.exec.expr.CodeGenerator;
import org.apache.drill.exec.rpc.user.UserSession;
import org.apache.drill.exec.server.options.SessionOptionManager;
import org.codehaus.commons.compiler.CompileException;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import org.objectweb.asm.ClassReader;
import org.objectweb.asm.ClassWriter;
import org.objectweb.asm.tree.ClassNode;
public class TestClassTransformation extends BaseTestQuery {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestClassTransformation.class);
private static final int ITERATION_COUNT = Integer.parseInt(System.getProperty("TestClassTransformation.iteration", "1"));
private static SessionOptionManager sessionOptions;
@BeforeClass
public static void beforeTestClassTransformation() {
// Tests here require the byte-code merge technique and are meaningless
// if the plain-old Java technique is selected. Force the plain-Java
// technique to be off if it happened to be set on in the default
// configuration.
System.setProperty(CodeCompiler.PREFER_POJ_CONFIG, "false");
final UserSession userSession = UserSession.Builder.newBuilder()
.withOptionManager(getDrillbitContext().getOptionManager())
.build();
sessionOptions = userSession.getOptions();
}
@Test
public void testJaninoClassCompiler() throws Exception {
logger.debug("Testing JaninoClassCompiler");
sessionOptions.setLocalOption(ClassCompilerSelector.JAVA_COMPILER_OPTION, ClassCompilerSelector.CompilerPolicy.JANINO.name());
for (int i = 0; i < ITERATION_COUNT; i++) {
compilationInnerClass(false); // Traditional byte-code manipulation
compilationInnerClass(true); // Plain-old Java
}
}
@Test
public void testJDKClassCompiler() throws Exception {
logger.debug("Testing JDKClassCompiler");
sessionOptions.setLocalOption(ClassCompilerSelector.JAVA_COMPILER_OPTION, ClassCompilerSelector.CompilerPolicy.JDK.name());
for (int i = 0; i < ITERATION_COUNT; i++) {
compilationInnerClass(false); // Traditional byte-code manipulation
compilationInnerClass(true); // Plain-old Java
}
}
@Test
public void testCompilationNoDebug() throws CompileException, ClassNotFoundException, ClassTransformationException, IOException {
CodeGenerator<ExampleInner> cg = newCodeGenerator(ExampleInner.class, ExampleTemplateWithInner.class);
ClassSet classSet = new ClassSet(null, cg.getDefinition().getTemplateClassName(), cg.getMaterializedClassName());
String sourceCode = cg.generateAndGet();
sessionOptions.setLocalOption(ClassCompilerSelector.JAVA_COMPILER_OPTION, ClassCompilerSelector.CompilerPolicy.JDK.name());
sessionOptions.setLocalOption(ClassCompilerSelector.JAVA_COMPILER_DEBUG_OPTION, false);
QueryClassLoader loader = new QueryClassLoader(config, sessionOptions);
byte[][] codeWithoutDebug = loader.getClassByteCode(classSet.generated, sourceCode);
loader.close();
int sizeWithoutDebug = 0;
for (byte[] bs : codeWithoutDebug) {
sizeWithoutDebug += bs.length;
}
sessionOptions.setLocalOption(ClassCompilerSelector.JAVA_COMPILER_DEBUG_OPTION, true);
loader = new QueryClassLoader(config, sessionOptions);
byte[][] codeWithDebug = loader.getClassByteCode(classSet.generated, sourceCode);
loader.close();
int sizeWithDebug = 0;
for (byte[] bs : codeWithDebug) {
sizeWithDebug += bs.length;
}
Assert.assertTrue("Debug code is smaller than optimized code!!!", sizeWithDebug > sizeWithoutDebug);
logger.debug("Optimized code is {}% smaller than debug code.", (int)((sizeWithDebug - sizeWithoutDebug)/(double)sizeWithDebug*100));
}
@Test // DRILL-6524
public void testScalarReplacementInCondition() throws Exception {
ClassTransformer.ClassNames classNames = new ClassTransformer.ClassNames("org.apache.drill.CompileClassWithIfs");
String entireClass = DrillFileUtils.getResourceAsString(DrillFileUtils.SEPARATOR + classNames.slash + ".java");
sessionOptions.setLocalOption(ClassCompilerSelector.JAVA_COMPILER_DEBUG_OPTION, false);
List<String> compilers = Arrays.asList(ClassCompilerSelector.CompilerPolicy.JANINO.name(),
ClassCompilerSelector.CompilerPolicy.JDK.name());
for (String compilerName : compilers) {
sessionOptions.setLocalOption(ClassCompilerSelector.JAVA_COMPILER_OPTION, compilerName);
QueryClassLoader queryClassLoader = new QueryClassLoader(config, sessionOptions);
byte[][] implementationClasses = queryClassLoader.getClassByteCode(classNames, entireClass);
ClassNode originalClass = AsmUtil.classFromBytes(implementationClasses[0], ClassReader.EXPAND_FRAMES);
ClassNode transformedClass = new ClassNode();
DrillCheckClassAdapter mergeGenerator = new DrillCheckClassAdapter(CompilationConfig.ASM_API_VERSION,
new CheckClassVisitorFsm(CompilationConfig.ASM_API_VERSION, transformedClass), true);
originalClass.accept(new ValueHolderReplacementVisitor(mergeGenerator, true));
if (!AsmUtil.isClassOk(logger, classNames.dot, transformedClass)) {
throw new IllegalStateException(String.format("Problem found after transforming %s", classNames.dot));
}
ClassWriter writer = new ClassWriter(ClassWriter.COMPUTE_FRAMES);
transformedClass.accept(writer);
byte[] outputClass = writer.toByteArray();
queryClassLoader.injectByteCode(classNames.dot, outputClass);
Class<?> transformedClazz = queryClassLoader.findClass(classNames.dot);
transformedClazz.getMethod("doSomething").invoke(null);
}
}
@Test
public void testScalarReplacementWithArrayAssignment() throws Exception {
ClassTransformer.ClassNames classNames = new ClassTransformer.ClassNames("org.apache.drill.CompileClassWithArraysAssignment");
String entireClass = DrillFileUtils.getResourceAsString(DrillFileUtils.SEPARATOR + classNames.slash + ".java");
sessionOptions.setLocalOption(ClassCompilerSelector.JAVA_COMPILER_DEBUG_OPTION, false);
List<String> compilers = Arrays.asList(ClassCompilerSelector.CompilerPolicy.JANINO.name(),
ClassCompilerSelector.CompilerPolicy.JDK.name());
for (String compilerName : compilers) {
sessionOptions.setLocalOption(ClassCompilerSelector.JAVA_COMPILER_OPTION, compilerName);
QueryClassLoader queryClassLoader = new QueryClassLoader(config, sessionOptions);
byte[][] implementationClasses = queryClassLoader.getClassByteCode(classNames, entireClass);
ClassNode originalClass = AsmUtil.classFromBytes(implementationClasses[0], ClassReader.EXPAND_FRAMES);
ClassNode transformedClass = new ClassNode();
DrillCheckClassAdapter mergeGenerator = new DrillCheckClassAdapter(CompilationConfig.ASM_API_VERSION,
new CheckClassVisitorFsm(CompilationConfig.ASM_API_VERSION, transformedClass), true);
originalClass.accept(new ValueHolderReplacementVisitor(mergeGenerator, true));
if (!AsmUtil.isClassOk(logger, classNames.dot, transformedClass)) {
throw new IllegalStateException(String.format("Problem found after transforming %s", classNames.dot));
}
ClassWriter writer = new ClassWriter(ClassWriter.COMPUTE_FRAMES);
transformedClass.accept(writer);
byte[] outputClass = writer.toByteArray();
queryClassLoader.injectByteCode(classNames.dot, outputClass);
Class<?> transformedClazz = queryClassLoader.findClass(classNames.dot);
transformedClazz.getMethod("doSomething").invoke(null);
}
}
@Test // DRILL-5683
public void testCaseWithColumnExprsOnView() throws Exception {
String sqlCreate =
"create table dfs.tmp.t1 as\n" +
"select r_regionkey, r_name, case when mod(r_regionkey, 3) > 0 then mod(r_regionkey, 3) else null end as flag\n" +
"from cp.`tpch/region.parquet`";
try {
test(sqlCreate);
String sql = "select * from dfs.tmp.t1 where NOT (flag IS NOT NULL)";
setSessionOption(ExecConstants.SCALAR_REPLACEMENT_OPTION, ClassTransformer.ScalarReplacementOption.ON.name());
List<String> compilers = Arrays.asList(ClassCompilerSelector.CompilerPolicy.JANINO.name(),
ClassCompilerSelector.CompilerPolicy.JDK.name());
for (String compilerName : compilers) {
setSessionOption(ClassCompilerSelector.JAVA_COMPILER_OPTION, compilerName);
testBuilder()
.sqlQuery(sql)
.unOrdered()
.baselineColumns("r_regionkey", "r_name", "flag")
.baselineValues(0, "AFRICA", null)
.baselineValues(3, "EUROPE", null)
.go();
}
} finally {
test("drop table if exists dfs.tmp.t1");
resetSessionOption(ExecConstants.SCALAR_REPLACEMENT_OPTION);
resetSessionOption(ClassCompilerSelector.JAVA_COMPILER_OPTION);
}
}
/**
* Do a test of a three level class to ensure that nested code generators works correctly.
*/
private void compilationInnerClass(boolean asPoj) throws Exception{
CodeGenerator<ExampleInner> cg = newCodeGenerator(ExampleInner.class, ExampleTemplateWithInner.class);
cg.preferPlainJava(asPoj);
CodeCompiler.CodeGenCompiler cc = new CodeCompiler.CodeGenCompiler(config, sessionOptions);
@SuppressWarnings("unchecked")
Class<? extends ExampleInner> c = (Class<? extends ExampleInner>) cc.generateAndCompile(cg);
ExampleInner t = c.newInstance();
t.doOutside();
t.doInsideOutside();
}
private <T, X extends T> CodeGenerator<T> newCodeGenerator(Class<T> iface, Class<X> impl) {
final TemplateClassDefinition<T> template = new TemplateClassDefinition<>(iface, impl);
CodeGenerator<T> cg = CodeGenerator.get(template, getDrillbitContext().getOptionManager());
cg.plainJavaCapable(true);
ClassGenerator<T> root = cg.getRoot();
root.setMappingSet(new MappingSet(new GeneratorMapping("doOutside", null, null, null)));
root.getSetupBlock().directStatement("System.out.println(\"outside\");");
ClassGenerator<T> inner = root.getInnerGenerator("TheInnerClass");
inner.setMappingSet(new MappingSet(new GeneratorMapping("doInside", null, null, null)));
inner.getSetupBlock().directStatement("System.out.println(\"inside\");");
ClassGenerator<T> doubleInner = inner.getInnerGenerator("DoubleInner");
doubleInner.setMappingSet(new MappingSet(new GeneratorMapping("doDouble", null, null, null)));
doubleInner.getSetupBlock().directStatement("System.out.println(\"double\");");
return cg;
}
}