fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java

// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.
// This file is copied from
// https://github.com/apache/impala/blob/branch-2.9.0/fe/src/main/java/org/apache/impala/InlineViewRef.java
// and modified by Doris

package org.apache.doris.analysis;

import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.InlineView;
import org.apache.doris.catalog.View;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
import org.apache.doris.nereids.parser.Dialect;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.rewrite.ExprRewriter;
import org.apache.doris.thrift.TNullSide;

import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.util.ArrayList;
import java.util.List;
import java.util.Set;

/**
 * An inline view is a query statement with an alias. Inline views can be parsed directly
 * from a query string or represent a reference to a local or catalog view.
 */
public class InlineViewRef extends TableRef {
    private static final Logger LOG = LogManager.getLogger(InlineViewRef.class);

    private static final String DEFAULT_TABLE_ALIAS_FOR_SPARK_SQL = "__auto_generated_subquery_name";

    // Catalog or local view that is referenced.
    // Null for inline views parsed directly from a query string.
    private final View view;

    // If not null, these will serve as the column labels for the inline view. This provides
    // a layer of separation between column labels visible from outside the inline view
    // and column labels used in the query definition. Either all or none of the column
    // labels must be overridden.
    private List<String> explicitColLabels;
    private List<List<String>> explicitSubColPath;

    // ///////////////////////////////////////
    // BEGIN: Members that need to be reset()

    // The select or union statement of the inline view
    private QueryStmt queryStmt;

    // queryStmt has its own analysis context
    private Analyzer inlineViewAnalyzer;

    // list of tuple ids materialized by queryStmt
    private final ArrayList<TupleId> materializedTupleIds = Lists.newArrayList();

    // Map inline view's output slots to the corresponding resultExpr of queryStmt.
    protected final ExprSubstitutionMap sMap;

    // Map inline view's output slots to the corresponding baseTblResultExpr of queryStmt.
    protected final ExprSubstitutionMap baseTblSmap;

    // When parsing a ddl of hive view, it does not contains any catalog info,
    // so we need to record it in Analyzer
    // otherwise some error will occurs when resolving TableRef later.
    protected String externalCtl;

    // END: Members that need to be reset()
    // ///////////////////////////////////////

    /**
     * C'tor for creating inline views parsed directly from the a query string.
     */
    public InlineViewRef(String alias, QueryStmt queryStmt) {
        super(null, alias);
        this.queryStmt = queryStmt;
        this.view = null;
        sMap = new ExprSubstitutionMap();
        baseTblSmap = new ExprSubstitutionMap();
    }

    public InlineViewRef(String alias, QueryStmt queryStmt, List<String> colLabels) {
        this(alias, queryStmt);
        explicitColLabels = Lists.newArrayList(colLabels);
        if (LOG.isDebugEnabled()) {
            LOG.debug("inline view explicitColLabels {}", explicitColLabels);
        }
    }

    /**
     * C'tor for creating inline views that replace a local or catalog view ref.
     */
    public InlineViewRef(View view, TableRef origTblRef) {
        super(origTblRef.getName(), origTblRef.getExplicitAlias());
        queryStmt = view.getQueryStmt().clone();
        if (view.isLocalView()) {
            queryStmt.reset();
        }
        this.view = view;
        sMap = new ExprSubstitutionMap();
        baseTblSmap = new ExprSubstitutionMap();
        setJoinAttrs(origTblRef);
        explicitColLabels = view.getColLabels();
        // Set implicit aliases if no explicit one was given.
        if (hasExplicitAlias()) {
            return;
        }
        // TODO(zc)
        // view_.getTableName().toString().toLowerCase(), view.getName().toLowerCase()
        if (view.isLocalView()) {
            aliases = new String[]{view.getName()};
        } else {
            aliases = new String[]{name.toString(), view.getName()};
        }
        if (origTblRef.getLateralViewRefs() != null) {
            lateralViewRefs = (ArrayList<LateralViewRef>) origTblRef.getLateralViewRefs().clone();
        }
    }

    protected InlineViewRef(InlineViewRef other) {
        super(other);
        queryStmt = other.queryStmt.clone();
        view = other.view;
        inlineViewAnalyzer = other.inlineViewAnalyzer;
        if (other.explicitColLabels != null) {
            explicitColLabels = Lists.newArrayList(other.explicitColLabels);
        }
        materializedTupleIds.addAll(other.materializedTupleIds);
        sMap = other.sMap.clone();
        baseTblSmap = other.baseTblSmap.clone();
    }

    public List<String> getExplicitColLabels() {
        return explicitColLabels;
    }

    public List<String> getColLabels() {
        if (explicitColLabels != null) {
            return explicitColLabels;
        }
        return queryStmt.getColLabels();
    }

    public List<List<String>> getSubColPath() {
        if (explicitSubColPath != null) {
            return explicitSubColPath;
        }
        return queryStmt.getSubColPath();
    }

    @Override
    public void reset() {
        super.reset();
        queryStmt.reset();
        inlineViewAnalyzer = null;
        materializedTupleIds.clear();
        sMap.clear();
        baseTblSmap.clear();
    }

    @Override
    public TableRef clone() {
        return new InlineViewRef(this);
    }

    public void setNeedToSql(boolean needToSql) {
        queryStmt.setNeedToSql(needToSql);
    }

    /**
     * Analyzes the inline view query block in a child analyzer of 'analyzer', creates
     * a new tuple descriptor for the inline view and registers auxiliary eq predicates
     * between the slots of that descriptor and the select list exprs of the inline view;
     * then performs join clause analysis.
     */
    @Override
    public void analyze(Analyzer analyzer) throws AnalysisException, UserException {
        if (isAnalyzed) {
            return;
        }

        if (view == null && !hasExplicitAlias()) {
            String dialect = ConnectContext.get().getSessionVariable().getSqlDialect();
            Dialect sqlDialect = Dialect.getByName(dialect);
            if (Dialect.SPARK != sqlDialect) {
                ErrorReport.reportAnalysisException(ErrorCode.ERR_DERIVED_MUST_HAVE_ALIAS);
            }
            hasExplicitAlias = true;
            aliases = new String[] { DEFAULT_TABLE_ALIAS_FOR_SPARK_SQL };
        }

        // Analyze the inline view query statement with its own analyzer
        inlineViewAnalyzer = new Analyzer(analyzer);
        inlineViewAnalyzer.setInlineView(true);
        if (hasExplicitAlias) {
            inlineViewAnalyzer.setExplicitViewAlias(aliases[0]);
        }
        queryStmt.analyze(inlineViewAnalyzer);
        correlatedTupleIds.addAll(queryStmt.getCorrelatedTupleIds(inlineViewAnalyzer));

        queryStmt.getMaterializedTupleIds(materializedTupleIds);
        if (view != null && !hasExplicitAlias() && !view.isLocalView()) {
            name = analyzer.getFqTableName(name);
            aliases = new String[] { name.toString(), view.getName() };
        }
        //TODO(chenhao16): fix TableName in Db.Table style
        // name.analyze(analyzer);
        desc = analyzer.registerTableRef(this);
        isAnalyzed = true;  // true now that we have assigned desc

        // For constant selects we materialize its exprs into a tuple.
        if (materializedTupleIds.isEmpty()) {
            Preconditions.checkState(queryStmt instanceof SelectStmt);
            Preconditions.checkState(((SelectStmt) queryStmt).getTableRefs().isEmpty());
            desc.setIsMaterialized(true);
            materializedTupleIds.add(desc.getId());
        }
        // create sMap and baseTblSmap and register auxiliary eq predicates between our
        // tuple descriptor's slots and our *unresolved* select list exprs;
        // we create these auxiliary predicates so that the analyzer can compute the value
        // transfer graph through this inline view correctly (ie, predicates can get
        // propagated through the view);
        // if the view stmt contains analytic functions, we cannot propagate predicates
        // into the view, unless the predicates are compatible with the analytic
        // function's partition by clause, because those extra filters
        // would alter the results of the analytic functions (see IMPALA-1243)
        // TODO: relax this a bit by allowing propagation out of the inline view (but
        // not into it)
        List<SlotDescriptor> slots = analyzer.changeSlotToNullableOfOuterJoinedTuples();
        if (LOG.isDebugEnabled()) {
            LOG.debug("inline view query {}", queryStmt.toSql());
        }
        for (int i = 0; i < getColLabels().size(); ++i) {
            String colName = getColLabels().get(i);
            if (LOG.isDebugEnabled()) {
                LOG.debug("inline view register {}", colName);
            }
            SlotDescriptor slotDesc = analyzer.registerColumnRef(getAliasAsName(),
                                            colName, getSubColPath().get(i));
            Expr colExpr = queryStmt.getResultExprs().get(i);
            if (queryStmt instanceof SelectStmt && ((SelectStmt) queryStmt).getValueList() != null) {
                ValueList valueList = ((SelectStmt) queryStmt).getValueList();
                for (int j = 0; j < valueList.getRows().size(); ++j) {
                    slotDesc.addSourceExpr(valueList.getRows().get(j).get(i));
                }
            } else {
                slotDesc.setSourceExpr(colExpr);
            }
            slotDesc.setIsNullable(slotDesc.getIsNullable() || colExpr.isNullable());
            SlotRef slotRef = new SlotRef(slotDesc);
            // to solve select * from (values(1, 2, 3), (4, 5, 6)) a returns only one row.
            if (slotDesc.getSourceExprs().size() == 1) {
                sMap.put(slotRef, colExpr);
                baseTblSmap.put(slotRef, queryStmt.getBaseTblResultExprs().get(i));
            }
            if (createAuxPredicate(colExpr)) {
                analyzer.createAuxEquivPredicate(new SlotRef(slotDesc), colExpr.clone());
            }
        }
        analyzer.changeSlotsToNotNullable(slots);
        if (LOG.isDebugEnabled()) {
            LOG.debug("inline view " + getUniqueAlias() + " smap: " + sMap.debugString());
            LOG.debug("inline view " + getUniqueAlias() + " baseTblSmap: " + baseTblSmap.debugString());
        }

        // analyzeLateralViewRefs
        analyzeLateralViewRef(analyzer);

        // Now do the remaining join analysis
        // In general, we should do analyze join before do RegisterColumnRef. However, We cannot move analyze join
        // before generate sMap and baseTblSmap, because generate sMap and baseTblSmap will register all column refs
        // in the inline view. If inline view is on right side of left semi join, exception will be thrown.
        // Instead, we do a little trick in RegisterColumnRef to avoid this problem.
        analyzeJoin(analyzer);
    }

    /**
     * Checks if an auxiliary predicate should be created for an expr. Returns False if the
     * inline view has a SELECT stmt with analytic functions and the expr is not in the
     * common partition exprs of all the analytic functions computed by this inline view.
     */
    public boolean createAuxPredicate(Expr e) {
        if (!(queryStmt instanceof SelectStmt)
                || !((SelectStmt) queryStmt).hasAnalyticInfo()) {
            return true;
        }
        AnalyticInfo analyticInfo = ((SelectStmt) queryStmt).getAnalyticInfo();
        return analyticInfo.getCommonPartitionExprs().contains(e);
    }

    /**
     * Create a non-materialized tuple descriptor in descTbl for this inline view.
     * This method is called from the analyzer when registering this inline view.
     */
    @Override
    public TupleDescriptor createTupleDescriptor(Analyzer analyzer) throws AnalysisException {
        // Create a fake catalog table for the inline view
        int numColLabels = getColLabels().size();
        Preconditions.checkState(numColLabels > 0);
        Set<String> columnSet = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
        List<Column> columnList = Lists.newArrayList();
        for (int i = 0; i < numColLabels; ++i) {
            // inline view select statement has been analyzed. Col label should be filled.
            Expr selectItemExpr = queryStmt.getResultExprs().get(i);
            // String colAlias = queryStmt.getColLabels().get(i);
            String colAlias = getColLabels().get(i);

            // inline view col cannot have duplicate name
            if (columnSet.contains(colAlias)) {
                throw new AnalysisException(
                        "Duplicated inline view column alias: '" + colAlias + "'" + " in inline view "
                                + "'" + getAlias() + "'");
            }

            columnSet.add(colAlias);
            columnList.add(new Column(colAlias, selectItemExpr.getType(),
                    false, null, selectItemExpr.isNullable(),
                    null, ""));
        }
        InlineView inlineView = (view != null) ? new InlineView(view, columnList)
                : new InlineView(getExplicitAlias(), columnList);

        // Create the non-materialized tuple and set the fake table in it.
        TupleDescriptor result = analyzer.getDescTbl().createTupleDescriptor();
        result.setIsMaterialized(false);
        result.setTable(inlineView);
        analyzer.registerInlineViewTupleId(result.getId());
        return result;
    }

    /**
     * Makes each rhs expr in sMap nullable if necessary by wrapping as follows:
     * IF(TupleIsNull(), NULL, rhs expr)
     * Should be called only if this inline view is a nullable side of an outer join.
     * <p/>
     * We need to make an rhs exprs nullable if it evaluates to a non-NULL value
     * when all of its contained SlotRefs evaluate to NULL.
     * For example, constant exprs need to be wrapped or an expr such as
     * 'case slotref is null then 1 else 2 end'
     */
    //
    // protected void makeOutputNullable(Analyzer analyzer) throws AnalysisException, InternalException {
    //     // Gather all unique rhs SlotRefs into rhsSlotRefs
    //     List<SlotRef> rhsSlotRefs = Lists.newArrayList();
    //     Expr.collectList(sMap.rhs, SlotRef.class, rhsSlotRefs);
    //     // Map for substituting SlotRefs with NullLiterals.
    //     Expr.SubstitutionMap nullSMap = new Expr.SubstitutionMap();
    //     for (SlotRef rhsSlotRef : rhsSlotRefs) {
    //         nullSMap.lhs.add(rhsSlotRef.clone());
    //         nullSMap.rhs.add(NullLiteral.create(rhsSlotRef.getType()));
    //     }
    //
    //     // Make rhs exprs nullable if necessary.
    //     for (int i = 0; i < sMap.rhs.size(); ++i) {
    //         List<Expr> params = Lists.newArrayList();
    //         if (!requiresNullWrapping(analyzer, sMap.rhs.get(i), nullSMap)) {
    //             continue;
    //         }
    //         params.add(new TupleIsNullPredicate(materializedTupleIds));
    //         params.add(NullLiteral.create(sMap.rhs.get(i).getType()));
    //         params.add(sMap.rhs.get(i));
    //         Expr ifExpr = new FunctionCallExpr("if", params);
    //         ifExpr.analyze(analyzer);
    //         sMap.rhs.set(i, ifExpr);
    //     }
    // }

    protected void makeOutputNullable(Analyzer analyzer) throws AnalysisException, UserException {
        try {
            makeOutputNullableHelper(analyzer, sMap);
            makeOutputNullableHelper(analyzer, baseTblSmap);
        } catch (Exception e) {
            // should never happen
            throw new IllegalStateException(e);
        }
    }

    protected void makeOutputNullableHelper(Analyzer analyzer, ExprSubstitutionMap smap)
            throws Exception {
        // Gather all unique rhs SlotRefs into rhsSlotRefs
        List<SlotRef> rhsSlotRefs = Lists.newArrayList();
        Expr.collectList(smap.getRhs(), SlotRef.class, rhsSlotRefs);
        // Map for substituting SlotRefs with NullLiterals.
        ExprSubstitutionMap nullSMap = new ExprSubstitutionMap();
        for (SlotRef rhsSlotRef : rhsSlotRefs) {
            nullSMap.put(rhsSlotRef.clone(), NullLiteral.create(rhsSlotRef.getType()));
        }


        // Make rhs exprs nullable if necessary.
        for (int i = 0; i < smap.getRhs().size(); ++i) {
            List<Expr> params = Lists.newArrayList();
            if (!requiresNullWrapping(analyzer, smap.getRhs().get(i), nullSMap)) {
                continue;
            }
            if (analyzer.isOuterJoinedLeftSide(materializedTupleIds.get(0))) {
                params.add(new TupleIsNullPredicate(materializedTupleIds, TNullSide.LEFT));
            } else {
                params.add(new TupleIsNullPredicate(materializedTupleIds, TNullSide.RIGHT));
            }
            params.add(NullLiteral.create(smap.getRhs().get(i).getType()));
            params.add(smap.getRhs().get(i));
            Expr ifExpr = new FunctionCallExpr("if", params);
            ifExpr.analyze(analyzer);
            smap.getRhs().set(i, ifExpr);
        }
    }

    /**
     * Replaces all SloRefs in expr with a NullLiteral using nullSMap, and evaluates the
     * resulting constant expr. Returns true if the constant expr yields a non-NULL value,
     * false otherwise.
     */
    private boolean requiresNullWrapping(Analyzer analyzer, Expr expr, ExprSubstitutionMap nullSMap)
            throws UserException {
        // If the expr is already wrapped in an IF(TupleIsNull(), NULL, expr)
        // then do not try to execute it.
        if (expr.contains(TupleIsNullPredicate.class)) {
            return true;
        }
        return true;
    }

    @Override
    public void rewriteExprs(ExprRewriter rewriter, Analyzer analyzer)
            throws AnalysisException {
        super.rewriteExprs(rewriter, analyzer);
        queryStmt.rewriteExprs(rewriter);
    }

    @Override
    public List<TupleId> getMaterializedTupleIds() {
        Preconditions.checkState(isAnalyzed);
        Preconditions.checkState(materializedTupleIds.size() > 0);
        return materializedTupleIds;
    }

    public QueryStmt getViewStmt() {
        return queryStmt;
    }

    public void setViewStmt(QueryStmt queryStmt) {
        this.queryStmt = queryStmt;
    }

    public Analyzer getAnalyzer() {
        Preconditions.checkState(isAnalyzed);
        return inlineViewAnalyzer;
    }

    public ExprSubstitutionMap getSmap() {
        Preconditions.checkState(isAnalyzed);
        return sMap;
    }

    public ExprSubstitutionMap getBaseTblSmap() {
        Preconditions.checkState(isAnalyzed);
        return baseTblSmap;
    }

    public boolean isLocalView() {
        return view == null || view.isLocalView();
    }

    public View getView() {
        return view;
    }

    public QueryStmt getQueryStmt() {
        return queryStmt;
    }

    public void setExternalCtl(String externalCtl) {
        this.externalCtl = externalCtl;
    }

    public String getExternalCtl() {
        return this.externalCtl;
    }

    @Override
    public String tableNameToSql() {
        // Enclose the alias in quotes if Hive cannot parse it without quotes.
        // This is needed for view compatibility between Impala and Hive.
        if (view != null) {
            // FIXME: this may result in a sql cache problem
            // See pr #6736 and issue #6735
            return super.tableNameToSql();
        }

        String aliasSql = null;
        String alias = getExplicitAlias();
        if (alias != null) {
            aliasSql = ToSqlUtils.getIdentSql(alias);
        }
        StringBuilder sb = new StringBuilder();
        sb.append("(").append(queryStmt.toSqlWithSelectList()).append(") ").append(aliasSql);
        return sb.toString();
    }

    @Override
    public String tableRefToDigest() {
        String aliasSql = null;
        String alias = getExplicitAlias();
        if (alias != null) {
            aliasSql = ToSqlUtils.getIdentSql(alias);
        }
        if (view != null) {
            return name.toSql() + (aliasSql == null ? "" : " " + aliasSql);
        }

        StringBuilder sb = new StringBuilder()
                .append("(")
                .append(queryStmt.toDigest())
                .append(") ")
                .append(aliasSql);

        return sb.toString();
    }
}