test/e2e/pig/tests/utf8.conf

#!/usr/bin/env perl
############################################################################
#  Licensed to the Apache Software Foundation (ASF) under one or more
#  contributor license agreements.  See the NOTICE file distributed with
#  this work for additional information regarding copyright ownership.
#  The ASF licenses this file to You under the Apache License, Version 2.0
#  (the "License"); you may not use this file except in compliance with
#  the License.  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

###############################################################################
# Nightly tests for pig.
#
#

$cfg = {
        'driver' => 'Pig',

        'groups' => [

                {
                'name' => 'Utf8Test',
                'tests' => [
                        {
                        'num' => 1,
# TC1.1 Execute Pig script with Chinese script name and path, filter by Chinese value, and output to Chinese folder
		,'pig' => q\#!/usr/bin/python
		# -*- coding: utf-8 -*-
from org.apache.pig.scripting import Pig
import os
#pig script dir
pig_script_dir=u":TMP:/脚本路径"
#delete if the dir already exists
if os.path.exists(pig_script_dir):
    os.remove(pig_script_dir)
#create the dir
os.mkdir(pig_script_dir)
#create pig script
pig_script = u":TMP:/脚本路径/超.pig"
pigfile = open( pig_script, "w+")
pigfile.write("""
A = LOAD ':INPATH:/utf8Data/学生/utf8Student' as (name:chararray, age:int, gpa:double);
B = filter A by name == '佛';
store B into ':OUTPATH:';
""")
pigfile.close()

#execute pig script

result = Pig.compileFromFile( pig_script ).bind().runSingle()

if result.isSuccessful():
    print "Pig job PASSED"
else:
    raise "Pig job FAILED"
\,

             'verify_pig_script' => q\A = LOAD ':INPATH:/utf8Data/学生/utf8Student' as (name:chararray, age:int, gpa:double);
                                      B = filter A by name == '佛';
                                      store B into ':OUTPATH:';
\
                        },
 #--------------------------------------------------------------------------------
                        {
                        'num' => 2,
# TC1.2 Joining tables on columns storing Chinese data in Pig
                        'pig' => q\
A = LOAD ':INPATH:/utf8Data/学生/utf8Student' as (name:chararray, age:int, gpa:double);
B = LOAD ':INPATH:/utf8Data/选民/utf8Voter' AS (name:chararray, age:int, registration:chararray, contributions:double);
C = JOIN A by name, B by name;
D = FOREACH C GENERATE gpa, registration, contributions;
store D into ':OUTPATH:';
\,
                        'verify_pig_script' => q\register :FUNCPATH:/testudf.jar;
A = LOAD ':INPATH:/utf8Data/学生/utf8Student' as (name:chararray, age:int, gpa:double);
B = LOAD ':INPATH:/utf8Data/选民/utf8Voter' AS (name:chararray, age:int, registration:chararray, contributions:double);
C = JOIN A by org.apache.pig.test.udf.evalfunc.ENCODE(name), B by org.apache.pig.test.udf.evalfunc.ENCODE(name);
D = FOREACH C GENERATE gpa, registration, contributions;
store D into ':OUTPATH:';
\,
                        }
#--------------------------------------------------------------------------------
                    ]  # end tests
                }, # end group
     ] # end groups
};