| #!/usr/bin/env perl |
| ############################################################################ |
| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| ############################################################################### |
| # Nightly tests for pig. |
| # |
| # |
| |
| $cfg = { |
| 'driver' => 'Pig', |
| |
| 'groups' => [ |
| |
| { |
| 'name' => 'Utf8Test', |
| 'tests' => [ |
| { |
| 'num' => 1, |
| # TC1.1 Execute Pig script with Chinese script name and path, filter by Chinese value, and output to Chinese folder |
| ,'pig' => q\#!/usr/bin/python |
| # -*- coding: utf-8 -*- |
| from org.apache.pig.scripting import Pig |
| import os |
| #pig script dir |
| pig_script_dir=u":TMP:/脚本路径" |
| #delete if the dir already exists |
| if os.path.exists(pig_script_dir): |
| os.remove(pig_script_dir) |
| #create the dir |
| os.mkdir(pig_script_dir) |
| #create pig script |
| pig_script = u":TMP:/脚本路径/超.pig" |
| pigfile = open( pig_script, "w+") |
| pigfile.write(""" |
| A = LOAD ':INPATH:/utf8Data/学生/utf8Student' as (name:chararray, age:int, gpa:double); |
| B = filter A by name == '佛'; |
| store B into ':OUTPATH:'; |
| """) |
| pigfile.close() |
| |
| #execute pig script |
| |
| result = Pig.compileFromFile( pig_script ).bind().runSingle() |
| |
| if result.isSuccessful(): |
| print "Pig job PASSED" |
| else: |
| raise "Pig job FAILED" |
| \, |
| |
| 'verify_pig_script' => q\A = LOAD ':INPATH:/utf8Data/学生/utf8Student' as (name:chararray, age:int, gpa:double); |
| B = filter A by name == '佛'; |
| store B into ':OUTPATH:'; |
| \ |
| }, |
| #-------------------------------------------------------------------------------- |
| { |
| 'num' => 2, |
| # TC1.2 Joining tables on columns storing Chinese data in Pig |
| 'pig' => q\ |
| A = LOAD ':INPATH:/utf8Data/学生/utf8Student' as (name:chararray, age:int, gpa:double); |
| B = LOAD ':INPATH:/utf8Data/选民/utf8Voter' AS (name:chararray, age:int, registration:chararray, contributions:double); |
| C = JOIN A by name, B by name; |
| D = FOREACH C GENERATE gpa, registration, contributions; |
| store D into ':OUTPATH:'; |
| \, |
| 'verify_pig_script' => q\register :FUNCPATH:/testudf.jar; |
| A = LOAD ':INPATH:/utf8Data/学生/utf8Student' as (name:chararray, age:int, gpa:double); |
| B = LOAD ':INPATH:/utf8Data/选民/utf8Voter' AS (name:chararray, age:int, registration:chararray, contributions:double); |
| C = JOIN A by org.apache.pig.test.udf.evalfunc.ENCODE(name), B by org.apache.pig.test.udf.evalfunc.ENCODE(name); |
| D = FOREACH C GENERATE gpa, registration, contributions; |
| store D into ':OUTPATH:'; |
| \, |
| } |
| #-------------------------------------------------------------------------------- |
| ] # end tests |
| }, # end group |
| ] # end groups |
| }; |